# Replication Archive for
# Coppock, Alexander. Generalizing from Survey Experiments Conducted on Mechanical Turk: A Replication Approach
# Forthcoming in Political Science Research and Methods

rm(list = ls())

# Uncomment to set working directory to replication archive
# setwd("")

# Uncomment to install packages
# install.packages("tidyverse")
# install.packages("coefplot")
# install.packages("devtools")
# devtools::install_github("acoppock/commarobust")

library(tidyverse)
library(commarobust)
library(coefplot)

# Load all datasets -------------------------------------------------------
load("coppock_generalizability_analysis_results.rdata")
load("coppock_generalizability_study_data.rdata")

results <-
  results %>%
  left_join(study_df) %>%
  left_join(dvs_df) %>%
  left_join(coefficients_df) %>%
  mutate(sample_factor = factor(
    sample,
    levels = c("original", "mt", "gfk"),
    labels = c("Original", "Mechanical Turk", "GfK")
  ))


# Facetted Graph ----------------------------------------------------------

facet_df <- 
  results %>%
  mutate(ui = est + 1.96*se,
         li = est - 1.96*se,
         entry = paste0(term, dv))

study_order <- 
  facet_df %>% 
  filter(sample == "original") %>%
  arrange(est)

facet_df <-
  facet_df %>%
  mutate(
    ui = est + 1.96*se,
    li = est - 1.96*se,
    entry = paste0(term, dv),
    entry = factor(entry, levels = study_order$entry),
    version =
      factor(
        sample,
        levels = c("gfk", "mt", "original"),
        labels = c("TESS/GfK", "Mechanical Turk", "Original")
      )
  )

facet_plot <- 
  ggplot(facet_df, aes(x = est, y = entry, group = version, color = version, shape = version)) +
  geom_point(position = position_dodgev(height = -.5), size = 3) +
  geom_errorbarh(aes(xmin = li, xmax = ui),
                 height = 0, position =  position_dodgev(height = -.5)) +
  geom_vline(xintercept = 0, linetype = "dashed", alpha = .5) + 
  xlab("Standardized Treatment Effect Estimates") + 
  scale_color_brewer(palette = "Dark2",  guide = guide_legend(reverse = TRUE)) +
  guides(size = guide_legend("Study Version"), 
         color = guide_legend("Study Version"), 
         shape = guide_legend("Study Version"))+
  facet_wrap( ~ study_factor, scales = "free", ncol = 3) +
  theme_bw() + 
  theme(legend.position = "bottom",
        legend.key.width = unit(4, "lines"),
        axis.text.y = element_blank(),
        axis.ticks.y = element_blank(),
        axis.title.y = element_blank(),
        strip.background = element_blank()) 

ggsave(filename = "coppock_generalizability_figure_2.pdf", facet_plot, height = 11, width = 8.5)



# Scatterplot -------------------------------------------------------------

ests <- 
  results %>%
  select(study, term, est, dv, sample) %>%
  spread(sample, est) 

ses <- 
  results %>%
  select(study, term, se, dv, sample) %>%
  spread(sample, se) 

ps <- 
  results %>%
  select(study, term, p, dv, sample) %>%
  spread(sample, p) 


scatter_df <- 
  left_join(ests, ses, 
            by = c("study", "term", "dv"), 
            suffix = c("_est", "_se")) %>%
  left_join(ps) %>%
  mutate(mt_sig = factor(mt <= 0.05, levels = c(T, F), labels = c("Significant", "Not Significant")),
         original_sig = factor(original <= 0.05, levels = c(T, F), labels = c("Significant", "Not Significant")),
         gfk_sig = factor(gfk <= 0.05, levels = c(T, F), labels = c("Significant", "Not Significant")),
         
         mt_ui = mt_est + 1.96*mt_se,
         mt_li = mt_est - 1.96*mt_se,
         
         original_ui = original_est + 1.96*original_se,
         original_li = original_est - 1.96*original_se,
         
         gfk_ui = gfk_est + 1.96*gfk_se,
         gfk_li = gfk_est - 1.96*gfk_se)


scatter_plot <- 
  ggplot(scatter_df, aes(x = mt_est, y = original_est, color = original_sig)) + 
  geom_point(aes(shape = original_sig)) +
  geom_rug() + 
  geom_segment(aes(x = mt_est, xend = mt_est, y = original_ui, yend = original_li), alpha = .2) + 
  geom_segment(aes(x = mt_ui, xend = mt_li, y = original_est, yend = original_est), alpha = .2) + 
  coord_cartesian(xlim = c(-1, 1), ylim = c(-1, 1)) +
  geom_hline(yintercept = 0, linetype = "dashed", alpha = .3) +
  geom_vline(xintercept = 0, linetype = "dashed", alpha = .3) +
  xlab("Mechanical Turk Version Standardized Estimate") + 
  ylab("Original Version Standardized Estimate") + 
  stat_smooth(aes(color = NULL, weight = 1 / mt_se ^ 2), 
              method = "lm", fullrange = TRUE, alpha = .1, color = "grey") + 
  theme_bw() +
  theme(legend.position = "bottom") +
  guides(color = guide_legend("Original Study"),
         shape = guide_legend("Original Study"))


ggsave(filename = "coppock_generalizability_figure_3.pdf", scatter_plot, height = 7, width = 7)

